#include <string>
#include <vector>
using namespace std;

#include "Tokenizer.hh"

Tokenizer::Tokenizer(){
}

Tokenizer::Tokenizer(string line){
  tokenize(line);
}

void Tokenizer::tokenize(string line){
  string delims = " \t";
  string::size_type last_pos = line.find_first_not_of(delims, 0);
  string::size_type pos = line.find_first_of(delims, last_pos);
  
  clear();
  while(string::npos != pos || string::npos != last_pos){
    push_back(line.substr(last_pos, pos - last_pos));
    last_pos = line.find_first_not_of(delims, pos);
    pos = line.find_first_of(delims, last_pos);
  }
}
